import gensim
import pandas as pd
from gensim.corpora import Dictionary, MmCorpus
from gensim.models.coherencemodel import CoherenceModel
from gensim import models
import pyLDAvis.gensim

dictionary = gensim.corpora.Dictionary.load('dictionary.dict')
corpus = gensim.corpora.MmCorpus('corpus.mm')
lda = gensim.models.ldamodel.LdaModel.load('model30.model')

lda_display = pyLDAvis.gensim.prepare(lda, corpus, dictionary, sort_topics=False)
pyLDAvis.show(lda_display)

cm = CoherenceModel(model=lda, corpus=corpus, dictionary=dictionary, coherence='u_mass')
coscore = cm.get_coherence()
print(coscore)

a = lda.show_topics(num_topics=30,formatted=False,num_words=10)
b = lda.top_topics(corpus,dictionary=dictionary,topn=10) # This orders the topics in the decreasing order of coherence score

topic2skillb = {}
topic2csb = {}
topic2skilla = {}
topic2csa = {}
num_topics =lda.num_topics
cnt =1

for ws in b:
    wset = set(w[1] for w in ws[0])
    topic2skillb[cnt] = wset
    topic2csb[cnt] = ws[1]
    cnt +=1

for ws in a:
    wset = set(w[0]for w in ws[1])
    topic2skilla[ws[0]+1] = wset

for i in range(1,num_topics+1):
    for j in range(1,num_topics+1):
        if topic2skilla[i].intersection(topic2skillb[j])==topic2skilla[i]:
            topic2csa[i] = topic2csb[j]

finalData = pd.DataFrame([],columns=['Topic','words'])
finalData['Topic']=topic2skilla.keys()
finalData['Topic'] = finalData['Topic'].apply(lambda x: 'Topic'+str(x))
finalData['words']=topic2skilla.values()
finalData['cs'] = topic2csa.values()
finalData.sort_values(by='cs',ascending=False,inplace=True)
finalData.to_csv('CoherenceScore.csv')
finalData
